{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:14:37.927557Z",
     "start_time": "2019-08-11T03:14:37.907681Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['set_series_1.csv',\n",
       " 'user_app_actived.csv',\n",
       " 'age_train.csv',\n",
       " 'age_test.csv',\n",
       " 'app_info.csv',\n",
       " 'train_data.csv',\n",
       " 'user_basic_info.csv',\n",
       " 'user_app_usage.csv',\n",
       " 'user_behavior_info.csv']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 导入相关库\n",
    "import os\n",
    "import pandas as pd\n",
    "from tqdm.autonotebook import *\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.metrics import accuracy_score\n",
    "import time\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from scipy.sparse import hstack\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from gensim.models import FastText, Word2Vec\n",
    "import re\n",
    "from keras.layers import *\n",
    "from keras.models import *\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.callbacks import *\n",
    "from keras.layers.advanced_activations import LeakyReLU, PReLU\n",
    "import keras.backend as K\n",
    "from keras.optimizers import *\n",
    "from keras.utils import to_categorical\n",
    "import tensorflow as tf\n",
    "import random as rn\n",
    "import gc\n",
    "import logging\n",
    "import gensim\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "# 显卡使用（如没显卡需要注释掉）\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = \"2,3\"\n",
    "np.random.seed(1024)\n",
    "rn.seed(1024)\n",
    "tf.set_random_seed(1024)\n",
    "path=\"data/\"\n",
    "os.listdir(\"data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:14:38.165097Z",
     "start_time": "2019-08-11T03:14:38.143963Z"
    }
   },
   "outputs": [],
   "source": [
    "# 读入数据（需加速）\n",
    "def get_age_data():\n",
    "    train_data = pd.read_csv(path + 'age_train.csv', header=None)\n",
    "    test_data = pd.read_csv(path + 'age_test.csv', header=None)\n",
    "    data = pd.concat([train_data, test_data], axis=0, sort=False).fillna(-1)\n",
    "    data.columns = ['uId', 'age_group']\n",
    "    return data\n",
    "\n",
    "def get_user_app_actived():\n",
    "    data = pd.read_csv(path + 'user_app_actived.csv', header=None)\n",
    "    data.columns = ['uId', 'appId']\n",
    "    return data\n",
    "\n",
    "def get_user_behavior_info():\n",
    "    data = pd.read_csv(path + 'user_behavior_info.csv', header=None)\n",
    "    data.columns = ['uId', 'bootTimes', 'AFuncTimes', 'BFuncTimes', 'CFuncTimes',\n",
    "                   'DFuncTimes', 'EFuncTimes', 'FFuncTimes', 'FFuncSum']\n",
    "    return data\n",
    "\n",
    "def get_user_basic_info():\n",
    "    data = pd.read_csv(path + 'user_basic_info.csv', header=None)\n",
    "    data.columns = ['uId', 'gender', 'city', 'prodName', 'ramCapacity', \n",
    "                   'ramLeftRation', 'romCapacity', 'romLeftRation', 'color',\n",
    "                   'fontSize', 'ct', 'carrier', 'os']\n",
    "    return data\n",
    "\n",
    "def get_app_info():\n",
    "    data = pd.read_csv(path + 'app_info.csv', header=None)\n",
    "    data.columns = ['appId', 'category']\n",
    "    return data\n",
    "\n",
    "# 测试的时候用True\n",
    "# 提特征改用False\n",
    "def get_user_app_usage(less_data=False):\n",
    "    if less_data:\n",
    "        reader = pd.read_csv(path + 'user_app_usage.csv', chunksize=2000000)\n",
    "        for i in reader:\n",
    "            data = i\n",
    "            break\n",
    "    else:\n",
    "        data = pd.read_csv(path + 'user_app_usage.csv', header=None)\n",
    "    data.columns = ['uId', 'appId', 'duration', 'times', 'use_date']\n",
    "    return data\n",
    "\n",
    "def get_time_usage():\n",
    "    data = pd.read_csv(path + 'set_series_1.csv', header=None)\n",
    "    data.columns = ['uId', 'timesUsage']\n",
    "    return data\n",
    "\n",
    "def get_usage_set():\n",
    "    data = pd.read_csv(path + 'nurbs_set.csv', header=None)\n",
    "    data.columns = ['uId', 'timeCount']\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:14:38.560648Z",
     "start_time": "2019-08-11T03:14:38.507911Z"
    }
   },
   "outputs": [],
   "source": [
    "# 需要用到的函数\n",
    "class AdamW(Optimizer):\n",
    "    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,  # decoupled weight decay (1/4)\n",
    "                 epsilon=1e-8, decay=0., **kwargs):\n",
    "        super(AdamW, self).__init__(**kwargs)\n",
    "        with K.name_scope(self.__class__.__name__):\n",
    "            self.iterations = K.variable(0, dtype='int64', name='iterations')\n",
    "            self.lr = K.variable(lr, name='lr')\n",
    "            self.beta_1 = K.variable(beta_1, name='beta_1')\n",
    "            self.beta_2 = K.variable(beta_2, name='beta_2')\n",
    "            self.decay = K.variable(decay, name='decay')\n",
    "            # decoupled weight decay (2/4)\n",
    "            self.wd = K.variable(weight_decay, name='weight_decay')\n",
    "        self.epsilon = epsilon\n",
    "        self.initial_decay = decay\n",
    "\n",
    "    @interfaces.legacy_get_updates_support\n",
    "    def get_updates(self, loss, params):\n",
    "        grads = self.get_gradients(loss, params)\n",
    "        self.updates = [K.update_add(self.iterations, 1)]\n",
    "        wd = self.wd  # decoupled weight decay (3/4)\n",
    "\n",
    "        lr = self.lr\n",
    "        if self.initial_decay > 0:\n",
    "            lr *= (1. / (1. + self.decay * K.cast(self.iterations,\n",
    "                                                  K.dtype(self.decay))))\n",
    "\n",
    "        t = K.cast(self.iterations, K.floatx()) + 1\n",
    "        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n",
    "                     (1. - K.pow(self.beta_1, t)))\n",
    "\n",
    "        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n",
    "        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n",
    "        self.weights = [self.iterations] + ms + vs\n",
    "\n",
    "        for p, g, m, v in zip(params, grads, ms, vs):\n",
    "            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n",
    "            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n",
    "            # decoupled weight decay (4/4)\n",
    "            p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) - lr * wd * p\n",
    "\n",
    "            self.updates.append(K.update(m, m_t))\n",
    "            self.updates.append(K.update(v, v_t))\n",
    "            new_p = p_t\n",
    "\n",
    "            # Apply constraints.\n",
    "            if getattr(p, 'constraint', None) is not None:\n",
    "                new_p = p.constraint(new_p)\n",
    "\n",
    "            self.updates.append(K.update(p, new_p))\n",
    "        return self.updates\n",
    "\n",
    "    def get_config(self):\n",
    "        config = {'lr': float(K.get_value(self.lr)),\n",
    "                  'beta_1': float(K.get_value(self.beta_1)),\n",
    "                  'beta_2': float(K.get_value(self.beta_2)),\n",
    "                  'decay': float(K.get_value(self.decay)),\n",
    "                  'weight_decay': float(K.get_value(self.wd)),\n",
    "                  'epsilon': self.epsilon}\n",
    "        base_config = super(AdamW, self).get_config()\n",
    "        return dict(list(base_config.items()) + list(config.items()))\n",
    "\n",
    "\n",
    "from keras.engine.topology import Layer\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:17:17.952533Z",
     "start_time": "2019-08-11T03:14:38.631052Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n",
      "nan\n"
     ]
    }
   ],
   "source": [
    "### 读入数据(想要五输入)\n",
    "id_label = get_age_data()\n",
    "active_data = get_user_app_actived()\n",
    "data = pd.merge(id_label, active_data, on='uId', how='left')\n",
    "data_info = get_app_info()\n",
    "cat_dict = dict(zip(data_info['appId'], data_info['category']))\n",
    "def get_review_data(row):\n",
    "    review = ''\n",
    "    try:\n",
    "        app_list = row['appId'].split('#')\n",
    "    except Exception:\n",
    "        app_list = ['0']\n",
    "        print(row['appId'])\n",
    "    for i in app_list:\n",
    "        try:\n",
    "            review += cat_dict[i] + ' '\n",
    "        except Exception:\n",
    "            review += '未知' + ' ' \n",
    "    return review\n",
    "data['appInfo'] = data.apply(lambda row:get_review_data(row), axis=1)\n",
    "time_usage_data = get_time_usage()\n",
    "data = pd.merge(data, time_usage_data, on='uId', how='left')\n",
    "data = data.fillna('未知')\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:17:17.993755Z",
     "start_time": "2019-08-11T03:17:17.954187Z"
    }
   },
   "outputs": [],
   "source": [
    "del id_label, active_data, data_info, cat_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:17:18.005700Z",
     "start_time": "2019-08-11T03:17:17.995869Z"
    }
   },
   "outputs": [],
   "source": [
    "### Tokenizer 序列化文本\n",
    "def set_tokenizer(docs, split_char=' ', max_len=100):\n",
    "    '''\n",
    "    输入\n",
    "    docs:文本列表\n",
    "    split_char:按什么字符切割\n",
    "    max_len:截取的最大长度\n",
    "    \n",
    "    输出\n",
    "    X:序列化后的数据\n",
    "    word_index:文本和数字对应的索引\n",
    "    '''\n",
    "    tokenizer = Tokenizer(lower=False, char_level=False, split=split_char)\n",
    "    tokenizer.fit_on_texts(docs)\n",
    "    X = tokenizer.texts_to_sequences(docs)\n",
    "    maxlen = max_len\n",
    "    X = pad_sequences(X, maxlen=maxlen, value=0)\n",
    "    word_index=tokenizer.word_index\n",
    "    return X, word_index\n",
    "\n",
    "### 做embedding 这里采用word2vec 可以换成其他例如（glove词向量）\n",
    "def trian_save_word2vec(docs, embed_size=300, save_name='w2v.txt', split_char=' '):\n",
    "    '''\n",
    "    输入\n",
    "    docs:输入的文本列表\n",
    "    embed_size:embed长度\n",
    "    save_name:保存的word2vec位置\n",
    "    \n",
    "    输出\n",
    "    w2v:返回的模型\n",
    "    '''\n",
    "    input_docs = []\n",
    "    for i in docs:\n",
    "        input_docs.append(i.split(split_char))\n",
    "    logging.basicConfig(\n",
    "    format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO)\n",
    "    w2v = Word2Vec(input_docs, size=embed_size, sg=1, window=8, seed=1017, workers=24, min_count=1, iter=10)\n",
    "    w2v.wv.save_word2vec_format(save_name)\n",
    "    print(\"w2v model done\")\n",
    "    return w2v\n",
    "\n",
    "# 得到embedding矩阵\n",
    "def get_embedding_matrix(word_index, embed_size=300, Emed_path=\"w2v_300.txt\"):\n",
    "    embeddings_index = gensim.models.KeyedVectors.load_word2vec_format(\n",
    "        Emed_path, binary=False)\n",
    "    nb_words = len(word_index)+1\n",
    "    embedding_matrix = np.zeros((nb_words, embed_size))\n",
    "    count = 0\n",
    "    for word, i in tqdm(word_index.items()):\n",
    "        if i >= nb_words:\n",
    "            continue\n",
    "        try:\n",
    "            embedding_vector = embeddings_index[word]\n",
    "        except:\n",
    "            embedding_vector = np.zeros(embed_size)\n",
    "            count += 1\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector    \n",
    "    print(\"null cnt\",count)\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:33:35.679359Z",
     "start_time": "2019-08-11T03:17:18.007484Z"
    }
   },
   "outputs": [],
   "source": [
    "# appId 90\n",
    "# timesUsage 600\n",
    "text_1_list = list(data['appId'])\n",
    "text_3_list = list(data['timesUsage'])\n",
    "\n",
    "del data['appId']\n",
    "del data['timesUsage']\n",
    "\n",
    "print('开始序列化')\n",
    "x1, index_1 = set_tokenizer(text_1_list, split_char='#', max_len=90)\n",
    "x3, index_3 = set_tokenizer(text_3_list, split_char=' ', max_len=600)\n",
    "print('序列化完成')\n",
    "gc.collect()\n",
    "\n",
    "np.save('tmp/x1.npy', x1)\n",
    "np.save('tmp/x3.npy', x3)\n",
    "np.save('tmp/x1.npy', index_1)\n",
    "np.save('tmp/x3.npy', index_3)\n",
    "\n",
    "# 值得提醒的是这个保存方法是采用w2v.wv.save_word2vec_format\n",
    "# 因此你如果载入自己训练模型的时候，需要载入后再按照这个函数来保存再在emed_path中输入\n",
    "trian_save_word2vec(text_1_list, save_name='w2v_model/cate_w2v_300.txt', split_char='#')\n",
    "gc.collect()\n",
    "trian_save_word2vec(text_3_list, save_name='w2v_model/w2v_300.txt', split_char=' ')\n",
    "gc.collect()\n",
    "\n",
    "# 得到emb矩阵\n",
    "emb1 = get_embedding_matrix(index_1, Emed_path='w2v_model/cate_w2v_300.txt')\n",
    "emb3 = get_embedding_matrix(index_3, Emed_path='w2v_model/w2v_300.txt')\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:33:53.477232Z",
     "start_time": "2019-08-11T03:33:35.681116Z"
    }
   },
   "outputs": [],
   "source": [
    "# 将feature作为输入进行处理(这个feature一定要替换掉nan与inf)\n",
    "# 这个feature的顺序一定要对应之前的feature的顺序，保证每条是对应的\n",
    "f1 = pd.read_csv('feature/f1.csv')\n",
    "f2 = pd.read_csv('feature/f2.csv')\n",
    "f3 = pd.read_csv('feature/f3.csv')\n",
    "f4 = pd.read_csv('feature/f4.csv')\n",
    "f5 = pd.read_csv('feature/f5.csv')\n",
    "\n",
    "feature = pd.concat([f1, f2, f3, f4, f5], axis=1, sort=False)\n",
    "feature = feature.fillna(-1)\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "ss=StandardScaler()\n",
    "ss.fit(feature)\n",
    "hin_feature = ss.transform(feature)\n",
    "num_feature_input = hin_feature.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:33:53.789323Z",
     "start_time": "2019-08-11T03:33:53.478841Z"
    }
   },
   "outputs": [],
   "source": [
    "# 区分开train和valid,test\n",
    "# 这里是假设三输入\n",
    "train_data = data[data['age_group']!=-1]\n",
    "train_input_1 = x1[:len(train_data)]\n",
    "test_input_1 = x1[len(train_data):]\n",
    "train_input_3 = x3[:len(train_data)]\n",
    "test_input_3 = x3[len(train_data):]\n",
    "train_input_5 = hin_feature[:len(train_data)]\n",
    "test_input_5 = hin_feature[len(train_data):]\n",
    "label = to_categorical(train_data['age_group'] - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-11T03:33:54.070279Z",
     "start_time": "2019-08-11T03:33:53.790972Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.initializers import *\n",
    "\n",
    "def model_conv(emb1, emb3, num_feature_input):\n",
    "    '''\n",
    "    注意这个inputs\n",
    "    seq1、seq2分别是两个输入\n",
    "    hin是feature层输入\n",
    "    是否做emb可选可不选，\n",
    "    这个就是我们之前训练已经得到的用于embedding的（embedding_matrix1， embedding_matrix2）\n",
    "    '''\n",
    "    K.clear_session()\n",
    "\n",
    "    emb_layer_1 = Embedding(\n",
    "        input_dim=emb1.shape[0],\n",
    "        output_dim=emb1.shape[1],\n",
    "        weights=[emb1],\n",
    "        input_length=90,\n",
    "        trainable=False\n",
    "    )\n",
    "    \n",
    "    emb_layer_3 = Embedding(\n",
    "        input_dim=emb3.shape[0],\n",
    "        output_dim=emb3.shape[1],\n",
    "        weights=[emb3],\n",
    "        input_length=600,\n",
    "        trainable=False\n",
    "    )\n",
    "    \n",
    "    \n",
    "    seq1 = Input(shape=(90,))\n",
    "    seq3 = Input(shape=(600,))    \n",
    "    \n",
    "    x1 = emb_layer_1(seq1)\n",
    "    x3 = emb_layer_3(seq3)\n",
    "    \n",
    "    sdrop=SpatialDropout1D(rate=0.2)\n",
    "\n",
    "    x1 = sdrop(x1)\n",
    "    x3 = sdrop(x3)\n",
    "    \n",
    "    x = Dropout(0.2)(Bidirectional(CuDNNLSTM(200, return_sequences=True))(x1))\n",
    "    semantic = TimeDistributed(Dense(100, activation=\"tanh\"))(x)\n",
    "    merged_1 = Lambda(lambda x: K.max(x, axis=1), output_shape=(100,))(semantic)\n",
    "    merged_1_avg = Lambda(lambda x: K.mean(x, axis=1), output_shape=(100,))(semantic)\n",
    "    \n",
    "    x = Dropout(0.2)(Bidirectional(CuDNNLSTM(200, return_sequences=True))(x3))\n",
    "    semantic = TimeDistributed(Dense(100, activation=\"tanh\"))(x)\n",
    "    merged_3 = Lambda(lambda x: K.max(x, axis=1), output_shape=(100,))(semantic)\n",
    "    merged_3_avg = Lambda(lambda x: K.mean(x, axis=1), output_shape=(100,))(semantic)\n",
    "    \n",
    "    hin = Input(shape=(num_feature_input, ))\n",
    "    htime = Dense(16, activation='relu')(hin)\n",
    "    \n",
    "    x = concatenate([merged_1, merged_3, merged_1_avg, merged_3_avg, htime])\n",
    "    \n",
    "    x = Dropout(0.2)(Activation(activation=\"relu\")(BatchNormalization()(Dense(1000)(x))))\n",
    "    x = Activation(activation=\"relu\")(BatchNormalization()(Dense(500)(x)))\n",
    "    pred = Dense(6, activation='softmax')(x)\n",
    "    model = Model(inputs=[seq1, seq3, hin], outputs=pred)\n",
    "    from keras.utils import multi_gpu_model\n",
    "    model = multi_gpu_model(model, 2)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=AdamW(lr=0.001,weight_decay=0.08,),metrics=[\"accuracy\"])\n",
    "    return model\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-08-11T03:14:39.713Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, random_state=1017, shuffle=True)\n",
    "sub = np.zeros((test_input_5.shape[0], 6))\n",
    "oof_pred = np.zeros((train_input_5.shape[0], 6))\n",
    "score = []\n",
    "count = 0\n",
    "if not os.path.exists(\"model\"):\n",
    "    os.mkdir(\"model\")\n",
    "\n",
    "for i, (train_index, test_index) in enumerate(skf.split(train_input_5, train_data['age_group'])):\n",
    "    print(\"FOLD | \", count+1)\n",
    "    print(\"###\"*35)\n",
    "    gc.collect()\n",
    "    filepath = \"model/nn_v1_%d.h5\" % count\n",
    "    checkpoint = ModelCheckpoint(\n",
    "        filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max',save_weights_only=True)\n",
    "    reduce_lr = ReduceLROnPlateau(\n",
    "        monitor='val_acc', factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n",
    "    earlystopping = EarlyStopping(\n",
    "        monitor='val_acc', min_delta=0.0001, patience=5, verbose=1, mode='max')\n",
    "    callbacks = [checkpoint, reduce_lr, earlystopping]\n",
    "    model_age = model_conv(emb1, emb3, num_feature_input)\n",
    "    if count==0:model_age.summary()\n",
    "    x1_tr, x1_va = np.array(train_input_1)[train_index], np.array(train_input_1)[test_index]    \n",
    "    x3_tr, x3_va = np.array(train_input_3)[train_index], np.array(train_input_3)[test_index]\n",
    "    x5_tr, x5_va = np.array(train_input_5)[train_index], np.array(train_input_5)[test_index]\n",
    "    y_tr, y_va = label[train_index], label[test_index]\n",
    "    \n",
    "    hist = model_age.fit([x1_tr, x3_tr, x5_tr],\n",
    "                         y_tr, batch_size=4096, epochs=50, \n",
    "                         validation_data=([x1_va, x3_va, x5_va], y_va),\n",
    "                         callbacks=callbacks, verbose=1, shuffle=True)\n",
    "\n",
    "    model_age.load_weights(filepath)\n",
    "    oof_pred[test_index] = model_age.predict([x1_va, x3_va, x5_va],batch_size=2048,verbose=1)\n",
    "    sub += model_age.predict([test_input_1, test_input_3, test_input_5],batch_size=2048,verbose=1)/skf.n_splits\n",
    "    score.append(np.max(hist.history['val_acc']))\n",
    "    count += 1\n",
    "print('acc:', np.mean(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-08-11T03:14:40.012Z"
    }
   },
   "outputs": [],
   "source": [
    "test = data[data['age_group'] == -1]\n",
    "submit = test[['uId']]\n",
    "submit.columns = ['id']\n",
    "submit['label'] = sub.argmax(1)+1\n",
    "if not os.path.exists(\"result\"):\n",
    "    os.mkdir(\"result\")\n",
    "submit.to_csv(\"./result/submission.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-08-11T03:14:40.347Z"
    }
   },
   "outputs": [],
   "source": [
    "oof = np.concatenate((oof_pred,sub))\n",
    "oof = pd.DataFrame(oof)\n",
    "oof.columns = [str(i+1) for i in range(6)]\n",
    "oof['id'] = pd.concat([train_data[['uId']],test[['uId']]])['uId'].values\n",
    "oof.to_csv(\"./result/v1_test.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-08-11T03:14:40.612Z"
    }
   },
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
